from scrapy.spider import BaseSpider
from scrapy.selector import HtmlXPathSelector
from scrapy.item import Item
from scrapy.http import Request, FormRequest, HtmlResponse

import time
import gtk
import webkit
import jswebkit


class BerkeleyEDUSpider(BaseSpider):
    name = 'BerkeleyEDU'
    allowed_domains = ['berkeley.edu']
    start_urls = ['http://webcast.berkeley.edu/series.html']

    # rules = (
    #     # Extract links matching 'category.php' (but not matching 'subsection.php')
    #     # and follow links from them (since no callback means follow=True by default).
    #     #Rule(SgmlLinkExtractor(allow=('category\.php', ), deny=('subsection\.php', ))),
    #
    #     # Extract links matching 'item.php' and parse them with the spider's method parse_item
    #     Rule(SgmlLinkExtractor(allow=('', )), callback='parse_videolist'),
    #     #Rule(SgmlLinkExtractor(allow=('playlist', )), callback='parse_item'),
    # )

    def parse(self, response):
        parsedRes = self.process_js(response)
        #self.shellDebug(parsedRes)
        self.log('Playlist page! %s' % parsedRes.url)

        hxs = HtmlXPathSelector(parsedRes)
        for div in hxs.select('//div[@id="course_list"]'):
            #print div.extract()
            for url in div.select('.//a/@href').extract():
                self.log('Playlist URLs! %s' % url)

    def process_js(self, response):
        self.log("Parsing JS")
        if(type(response) is not FormRequest):
            webview = webkit.WebView()
            webview.connect( 'load-finished', lambda v,f: gtk.main_quit() )
            webview.load_uri(response.url)
            gtk.main()
            body = response.body
            renderedBody = body
            while True:
                js = jswebkit.JSContext( webview.get_main_frame().get_global_context())
                renderedBody = str(js.EvaluateScript('document.documentElement.innerHTML'))
                if body != renderedBody:
                    break

            return response.replace(body=renderedBody)

    def shellDebug(self, response):
        from scrapy.shell import inspect_response
        inspect_response(response)